from __future__ import division
from operator import itemgetter, attrgetter
import os
import sys
import math

# step1:
queryTermFreqDict = {}
inputAuxFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/realFreqOfTermsInQueries_head_99K_sortedByQueryTermFreq.txt"
inputAuxFileHandler = open(inputAuxFileName,"r")

for line in inputAuxFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    queryTermFreq = int(lineElements[1])
    if queryTerm not in queryTermFreqDict:
        queryTermFreqDict[queryTerm] = queryTermFreq

print "len(queryTermFreqDict):",len(queryTermFreqDict)
inputAuxFileHandler.close()

# step2:
# key: originalFreqR
# value: a tuple with the following format (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
smoothingInfoDict = {}

inputAuxFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/originalFreqWithNewFreqAdded_ProbabilityAdded.txt"
inputAuxFileHandler = open(inputAuxFileName,"r")

# ignore 4 lines of meta data
inputAuxFileHandler.readline()
inputAuxFileHandler.readline()
inputAuxFileHandler.readline()
inputAuxFileHandler.readline()

for line in inputAuxFileHandler.readlines():
    lineElements = line.strip().split(" ")
    
    originalFreqR = int( lineElements[0] )
    freqOfFreqNr = int( lineElements[1] )
    modifiedFreqRStar = float( lineElements[2] )
    
    VFProbability = float( lineElements[3] )
    FProbability = float( lineElements[4] )
    MProbability = float( lineElements[5] )
    NFProbability = float( lineElements[6] )
    VRProbability = float( lineElements[7] )
    
    valueTuple = (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
    
    if originalFreqR not in smoothingInfoDict:
        smoothingInfoDict[originalFreqR] = valueTuple 

print "len(smoothingInfoDict):",len(smoothingInfoDict)
inputAuxFileHandler.close()

# step3:
# ranges setting for the length of each inverted list are as follows:
# 20% very frequent(VF): [3636098, the longest length)
# 20% frequent(F): [1369057,3636098)
# 20% mediumn(M): [560494,1369057)
# 20% not frequent(NF): [113617 - 560494)
# 20% rare(R): [1,113617) 

UPPER_BOUND_FOR_RANGE1 = 113617
UPPER_BOUND_FOR_RANGE2 = 560494
UPPER_BOUND_FOR_RANGE3 = 1369057
UPPER_BOUND_FOR_RANGE4 = 3636098


inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithTermFreqInCollection_head_10.txt"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithTermFreqInCollection_TermFreqInQLAdded_ProbabilityInQLAdded_DesidingFactorAdded_head_10.txt"
outputFileHandler = open(outputFileName,"w")

dataLine = inputFileHandler.readline()

while dataLine:
    dataLineElements = dataLine.strip().split(" ")
    lexiconTerm = dataLineElements[0]
    lengthOfListForLexiconTerm = int( dataLineElements[1] )
    
    if lexiconTerm in queryTermFreqDict:
        # I can get the freq from the queryTermFreqDict
        freqForCurrentLexiconTermInQL = queryTermFreqDict[lexiconTerm]
    else:
        # unseen terms but in the lexicon
        freqForCurrentLexiconTermInQL = 0
        
    valueTuple = smoothingInfoDict[freqForCurrentLexiconTermInQL]
    probabilityForThisTermToBeInQL = 0.0
    probabilityClass = "N/A"
    
    if freqForCurrentLexiconTermInQL <= 20:
        if lengthOfListForLexiconTerm >= 1 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE1:
            # it is very rare
            # example:
            # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
            probabilityClass = "VR"

        elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE1 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE2:
            # it is not frequent
            # example:
            # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
            probabilityClass = "NF"

        elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE2 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE3:
            # it is medium
            # example:
            # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
            probabilityClass = "M"

        elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE3 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE4:
            # it is frequent
            # example:
            # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
            probabilityClass = "F"

        elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE4:
            # it is very frequent
            # example:
            # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
            probabilityClass = "VF"        
    else:
        probabilityClass = "VF"
        (_,_,VFProbability,_,_,_,_) = valueTuple
    
    
    if probabilityClass == "VR":
        (_,_,_,_,_,_,VRProbability) = valueTuple
        probabilityForThisTermToBeInQL = VRProbability
    elif probabilityClass == "NF":
        (_,_,_,_,_,NFProbability,_) = valueTuple
        probabilityForThisTermToBeInQL = NFProbability
    elif probabilityClass == "M":
        (_,_,_,_,MProbability,_,_) = valueTuple
        probabilityForThisTermToBeInQL = MProbability
    elif probabilityClass == "F":
        (_,_,_,FProbability,_,_,_) = valueTuple
        probabilityForThisTermToBeInQL = FProbability
    elif probabilityClass == "VF":
        (_,_,VFProbability,_,_,_,_) = valueTuple
        probabilityForThisTermToBeInQL = VFProbability
    
    print lexiconTerm,freqForCurrentLexiconTermInQL,lengthOfListForLexiconTerm, probabilityClass,probabilityForThisTermToBeInQL
    outputFileHandler.write( dataLine.strip() + " " + str( probabilityForThisTermToBeInQL ) + " " + "\n")
    dataLine = inputFileHandler.readline()
    
inputFileHandler.close()
outputFileHandler.close()


